{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n",
      "/home/kain/Workstation/PyEnv/lib/python3.5/site-packages/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gc\n",
    "import time\n",
    "import category_encoders as ce\n",
    "from contextlib import contextmanager\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import category_encoders as ce\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from scipy.cluster.vq import kmeans2, whiten\n",
    "from sklearn.decomposition import truncated_svd\n",
    "from catboost import CatBoostClassifier\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "num_rows = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# One-hot encoding for categorical columns with get_dummies\n",
    "def one_hot_encoder(df, nan_as_category = True):\n",
    "    original_columns = list(df.columns)\n",
    "    categorical_columns = [col for col in df.columns if df[col].dtype == 'object']\n",
    "    df = pd.get_dummies(df, columns= categorical_columns, dummy_na= nan_as_category)\n",
    "    new_columns = [c for c in df.columns if c not in original_columns]\n",
    "    return df, new_columns\n",
    "\n",
    "# Preprocess application_train.csv and application_test.csv\n",
    "def application_train_test(num_rows = None, nan_as_category = False):\n",
    "    # Read data and merge\n",
    "    df = pd.read_csv('../data/application_train.csv', nrows= num_rows)\n",
    "    test_df = pd.read_csv('../data/application_test.csv', nrows= num_rows)\n",
    "    print(\"Train samples: {}, test samples: {}\".format(len(df), len(test_df)))\n",
    "    df = df.append(test_df).reset_index()\n",
    "    # Optional: Remove 4 applications with XNA CODE_GENDER (train set)\n",
    "#     df = df[df['CODE_GENDER'] != 'XNA']\n",
    "    \n",
    "    docs = [_f for _f in df.columns if 'FLAG_DOC' in _f]\n",
    "    live = [_f for _f in df.columns if ('FLAG_' in _f) & ('FLAG_DOC' not in _f) & ('_FLAG_' not in _f)]\n",
    "    \n",
    "    # NaN values for DAYS_EMPLOYED: 365.243 -> nan\n",
    "    df['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)\n",
    "\n",
    "    inc_by_org = df[['AMT_INCOME_TOTAL', 'ORGANIZATION_TYPE']].groupby('ORGANIZATION_TYPE').median()['AMT_INCOME_TOTAL']\n",
    "\n",
    "    df['NEW_CREDIT_TO_ANNUITY_RATIO'] = df['AMT_CREDIT'] / df['AMT_ANNUITY']\n",
    "    df['NEW_CREDIT_TO_GOODS_RATIO'] = df['AMT_CREDIT'] / df['AMT_GOODS_PRICE']\n",
    "    df['NEW_DOC_IND_KURT'] = df[docs].kurtosis(axis=1)\n",
    "    df['NEW_LIVE_IND_SUM'] = df[live].sum(axis=1)\n",
    "    df['NEW_INC_PER_CHLD'] = df['AMT_INCOME_TOTAL'] / (1 + df['CNT_CHILDREN'])\n",
    "    df['NEW_INC_BY_ORG'] = df['ORGANIZATION_TYPE'].map(inc_by_org)\n",
    "    df['NEW_EMPLOY_TO_BIRTH_RATIO'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n",
    "    df['NEW_ANNUITY_TO_INCOME_RATIO'] = df['AMT_ANNUITY'] / (1 + df['AMT_INCOME_TOTAL'])\n",
    "    df['NEW_SOURCES_PROD'] = df['EXT_SOURCE_1'] * df['EXT_SOURCE_2'] * df['EXT_SOURCE_3']\n",
    "    df['NEW_EXT_SOURCES_MEAN'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].mean(axis=1)\n",
    "    df['NEW_SCORES_STD'] = df[['EXT_SOURCE_1', 'EXT_SOURCE_2', 'EXT_SOURCE_3']].std(axis=1)\n",
    "    df['NEW_SCORES_STD'] = df['NEW_SCORES_STD'].fillna(df['NEW_SCORES_STD'].mean())\n",
    "    df['NEW_CAR_TO_BIRTH_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_BIRTH']\n",
    "    df['NEW_CAR_TO_EMPLOY_RATIO'] = df['OWN_CAR_AGE'] / df['DAYS_EMPLOYED']\n",
    "    df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_BIRTH']\n",
    "    df['NEW_PHONE_TO_BIRTH_RATIO'] = df['DAYS_LAST_PHONE_CHANGE'] / df['DAYS_EMPLOYED']\n",
    "    df['NEW_CREDIT_TO_INCOME_RATIO'] = df['AMT_CREDIT'] / df['AMT_INCOME_TOTAL']\n",
    "    \n",
    "    # Categorical features with Binary encode (0 or 1; two categories)\n",
    "    for bin_feature in ['CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY']:\n",
    "        df[bin_feature], uniques = pd.factorize(df[bin_feature])\n",
    "    # Categorical features with One-Hot encode\n",
    "    df, cat_cols = one_hot_encoder(df, nan_as_category)\n",
    "    \n",
    "    # Some simple new features (percentages)\n",
    "    # df['DAYS_EMPLOYED_PERC'] = df['DAYS_EMPLOYED'] / df['DAYS_BIRTH']\n",
    "    # df['INCOME_CREDIT_PERC'] = df['AMT_INCOME_TOTAL'] / df['AMT_CREDIT']\n",
    "    # df['INCOME_PER_PERSON'] = df['AMT_INCOME_TOTAL'] / df['CNT_FAM_MEMBERS']\n",
    "    # df['ANNUITY_INCOME_PERC'] = df['AMT_ANNUITY'] / df['AMT_INCOME_TOTAL']\n",
    "    # df['PAYMENT_RATE'] = df['AMT_ANNUITY'] / df['AMT_CREDIT']\n",
    "    del test_df\n",
    "    gc.collect()\n",
    "    return df\n",
    "\n",
    "# Preprocess bureau.csv and bureau_balance.csv\n",
    "def bureau_and_balance(num_rows = None, nan_as_category = True):\n",
    "    bureau = pd.read_csv('../data/bureau.csv', nrows = num_rows)\n",
    "    bb = pd.read_csv('../data/bureau_balance.csv', nrows = num_rows)\n",
    "    bb, bb_cat = one_hot_encoder(bb, nan_as_category)\n",
    "    bureau, bureau_cat = one_hot_encoder(bureau, nan_as_category)\n",
    "    \n",
    "    # Bureau balance: Perform aggregations and merge with bureau.csv\n",
    "    bb_aggregations = {'MONTHS_BALANCE': ['min', 'max', 'size']}\n",
    "    for col in bb_cat:\n",
    "        bb_aggregations[col] = ['mean']\n",
    "    bb_agg = bb.groupby('SK_ID_BUREAU').agg(bb_aggregations)\n",
    "    bb_agg.columns = pd.Index([e[0] + \"_\" + e[1].upper() for e in bb_agg.columns.tolist()])\n",
    "    bureau = bureau.join(bb_agg, how='left', on='SK_ID_BUREAU')\n",
    "    bureau.drop(['SK_ID_BUREAU'], axis=1, inplace= True)\n",
    "    del bb, bb_agg\n",
    "    gc.collect()\n",
    "    \n",
    "    # Bureau and bureau_balance numeric features\n",
    "    num_aggregations = {\n",
    "        'DAYS_CREDIT': [ 'mean', 'var'],\n",
    "        'DAYS_CREDIT_ENDDATE': [ 'mean'],\n",
    "        'DAYS_CREDIT_UPDATE': ['mean'],\n",
    "        'CREDIT_DAY_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_MAX_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM': [ 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_DEBT': [ 'mean', 'sum'],\n",
    "        'AMT_CREDIT_SUM_OVERDUE': ['mean'],\n",
    "        'AMT_CREDIT_SUM_LIMIT': ['mean', 'sum'],\n",
    "        'AMT_ANNUITY': ['max', 'mean'],\n",
    "        'CNT_CREDIT_PROLONG': ['sum'],\n",
    "        'MONTHS_BALANCE_MIN': ['min'],\n",
    "        'MONTHS_BALANCE_MAX': ['max'],\n",
    "        'MONTHS_BALANCE_SIZE': ['mean', 'sum']\n",
    "    }\n",
    "    # Bureau and bureau_balance categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in bureau_cat: cat_aggregations[cat] = ['mean']\n",
    "    for cat in bb_cat: cat_aggregations[cat + \"_MEAN\"] = ['mean']\n",
    "    \n",
    "    bureau_agg = bureau.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    bureau_agg.columns = pd.Index(['BURO_' + e[0] + \"_\" + e[1].upper() for e in bureau_agg.columns.tolist()])\n",
    "    # Bureau: Active credits - using only numerical aggregations\n",
    "    active = bureau[bureau['CREDIT_ACTIVE_Active'] == 1]\n",
    "    active_agg = active.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    active_agg.columns = pd.Index(['ACTIVE_' + e[0] + \"_\" + e[1].upper() for e in active_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(active_agg, how='left', on='SK_ID_CURR')\n",
    "    del active, active_agg\n",
    "    gc.collect()\n",
    "    # Bureau: Closed credits - using only numerical aggregations\n",
    "    closed = bureau[bureau['CREDIT_ACTIVE_Closed'] == 1]\n",
    "    closed_agg = closed.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    closed_agg.columns = pd.Index(['CLOSED_' + e[0] + \"_\" + e[1].upper() for e in closed_agg.columns.tolist()])\n",
    "    bureau_agg = bureau_agg.join(closed_agg, how='left', on='SK_ID_CURR')\n",
    "    del closed, closed_agg, bureau\n",
    "    gc.collect()\n",
    "    return bureau_agg\n",
    "\n",
    "# Preprocess previous_applications.csv\n",
    "def previous_applications(num_rows = None, nan_as_category = True):\n",
    "    prev = pd.read_csv('../data/previous_application.csv', nrows = num_rows)\n",
    "    prev, cat_cols = one_hot_encoder(prev, nan_as_category= True)\n",
    "    # Days 365.243 values -> nan\n",
    "    prev['DAYS_FIRST_DRAWING'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_FIRST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE_1ST_VERSION'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_LAST_DUE'].replace(365243, np.nan, inplace= True)\n",
    "    prev['DAYS_TERMINATION'].replace(365243, np.nan, inplace= True)\n",
    "    # Add feature: value ask / value received percentage\n",
    "    prev['APP_CREDIT_PERC'] = prev['AMT_APPLICATION'] / prev['AMT_CREDIT']\n",
    "    # Previous applications numeric features\n",
    "    num_aggregations = {\n",
    "        'AMT_ANNUITY': [ 'max', 'mean'],\n",
    "        'AMT_APPLICATION': ['min', 'mean'],\n",
    "        'AMT_CREDIT': ['min', 'max', 'mean'],\n",
    "        'APP_CREDIT_PERC': ['min', 'max', 'mean'],\n",
    "        'AMT_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "        'AMT_GOODS_PRICE': ['min', 'max', 'mean'],\n",
    "        'HOUR_APPR_PROCESS_START': ['min', 'max', 'mean'],\n",
    "        'RATE_DOWN_PAYMENT': ['min', 'max', 'mean'],\n",
    "        'DAYS_DECISION': ['min', 'max', 'mean'],\n",
    "        'CNT_PAYMENT': ['mean', 'sum'],\n",
    "    }\n",
    "    # Previous applications categorical features\n",
    "    cat_aggregations = {}\n",
    "    for cat in cat_cols:\n",
    "        cat_aggregations[cat] = ['mean']\n",
    "    \n",
    "    prev_agg = prev.groupby('SK_ID_CURR').agg({**num_aggregations, **cat_aggregations})\n",
    "    prev_agg.columns = pd.Index(['PREV_' + e[0] + \"_\" + e[1].upper() for e in prev_agg.columns.tolist()])\n",
    "    # Previous Applications: Approved Applications - only numerical features\n",
    "    approved = prev[prev['NAME_CONTRACT_STATUS_Approved'] == 1]\n",
    "    approved_agg = approved.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    approved_agg.columns = pd.Index(['APPROVED_' + e[0] + \"_\" + e[1].upper() for e in approved_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(approved_agg, how='left', on='SK_ID_CURR')\n",
    "    # Previous Applications: Refused Applications - only numerical features\n",
    "    refused = prev[prev['NAME_CONTRACT_STATUS_Refused'] == 1]\n",
    "    refused_agg = refused.groupby('SK_ID_CURR').agg(num_aggregations)\n",
    "    refused_agg.columns = pd.Index(['REFUSED_' + e[0] + \"_\" + e[1].upper() for e in refused_agg.columns.tolist()])\n",
    "    prev_agg = prev_agg.join(refused_agg, how='left', on='SK_ID_CURR')\n",
    "    del refused, refused_agg, approved, approved_agg, prev\n",
    "    gc.collect()\n",
    "    return prev_agg\n",
    "\n",
    "# Preprocess POS_CASH_balance.csv\n",
    "def pos_cash(num_rows = None, nan_as_category = True):\n",
    "    pos = pd.read_csv('../data/POS_CASH_balance.csv', nrows = num_rows)\n",
    "    pos, cat_cols = one_hot_encoder(pos, nan_as_category= True)\n",
    "    # Features\n",
    "    aggregations = {\n",
    "        'MONTHS_BALANCE': ['max', 'mean', 'size'],\n",
    "        'SK_DPD': ['max', 'mean'],\n",
    "        'SK_DPD_DEF': ['max', 'mean']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    \n",
    "    pos_agg = pos.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    pos_agg.columns = pd.Index(['POS_' + e[0] + \"_\" + e[1].upper() for e in pos_agg.columns.tolist()])\n",
    "    # Count pos cash accounts\n",
    "    pos_agg['POS_COUNT'] = pos.groupby('SK_ID_CURR').size()\n",
    "    del pos\n",
    "    gc.collect()\n",
    "    return pos_agg\n",
    "    \n",
    "# Preprocess installments_payments.csv\n",
    "def installments_payments(num_rows = None, nan_as_category = True):\n",
    "    ins = pd.read_csv('../data/installments_payments.csv', nrows = num_rows)\n",
    "    ins, cat_cols = one_hot_encoder(ins, nan_as_category= True)\n",
    "    # Percentage and difference paid in each installment (amount paid and installment value)\n",
    "    ins['PAYMENT_PERC'] = ins['AMT_PAYMENT'] / ins['AMT_INSTALMENT']\n",
    "    ins['PAYMENT_DIFF'] = ins['AMT_INSTALMENT'] - ins['AMT_PAYMENT']\n",
    "    # Days past due and days before due (no negative values)\n",
    "    ins['DPD'] = ins['DAYS_ENTRY_PAYMENT'] - ins['DAYS_INSTALMENT']\n",
    "    ins['DBD'] = ins['DAYS_INSTALMENT'] - ins['DAYS_ENTRY_PAYMENT']\n",
    "    ins['DPD'] = ins['DPD'].apply(lambda x: x if x > 0 else 0)\n",
    "    ins['DBD'] = ins['DBD'].apply(lambda x: x if x > 0 else 0)\n",
    "    # Features: Perform aggregations\n",
    "    aggregations = {\n",
    "        'NUM_INSTALMENT_VERSION': ['nunique'],\n",
    "        'DPD': ['max', 'mean', 'sum'],\n",
    "        'DBD': ['max', 'mean', 'sum'],\n",
    "        'PAYMENT_PERC': [ 'mean', 'sum', 'var'],\n",
    "        'PAYMENT_DIFF': [ 'mean', 'sum', 'var'],\n",
    "        'AMT_INSTALMENT': ['max', 'mean', 'sum'],\n",
    "        'AMT_PAYMENT': ['min', 'max', 'mean', 'sum'],\n",
    "        'DAYS_ENTRY_PAYMENT': ['max', 'mean', 'sum']\n",
    "    }\n",
    "    for cat in cat_cols:\n",
    "        aggregations[cat] = ['mean']\n",
    "    ins_agg = ins.groupby('SK_ID_CURR').agg(aggregations)\n",
    "    ins_agg.columns = pd.Index(['INSTAL_' + e[0] + \"_\" + e[1].upper() for e in ins_agg.columns.tolist()])\n",
    "    # Count installments accounts\n",
    "    ins_agg['INSTAL_COUNT'] = ins.groupby('SK_ID_CURR').size()\n",
    "    del ins\n",
    "    gc.collect()\n",
    "    return ins_agg\n",
    "\n",
    "# Preprocess credit_card_balance.csv\n",
    "def credit_card_balance(num_rows = None, nan_as_category = True):\n",
    "    cc = pd.read_csv('../data/credit_card_balance.csv', nrows = num_rows)\n",
    "    cc, cat_cols = one_hot_encoder(cc, nan_as_category= True)\n",
    "    # General aggregations\n",
    "    cc.drop(['SK_ID_PREV'], axis= 1, inplace = True)\n",
    "    cc_agg = cc.groupby('SK_ID_CURR').agg(['min', 'max', 'mean', 'sum', 'var'])\n",
    "    cc_agg.columns = pd.Index(['CC_' + e[0] + \"_\" + e[1].upper() for e in cc_agg.columns.tolist()])\n",
    "    # Count credit card lines\n",
    "    cc_agg['CC_COUNT'] = cc.groupby('SK_ID_CURR').size()\n",
    "    del cc\n",
    "    gc.collect()\n",
    "    return cc_agg\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train samples: 307511, test samples: 48744\n"
     ]
    }
   ],
   "source": [
    "num_rows = None\n",
    "df = application_train_test(num_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 258)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bureau df shape: (305811, 95)\n",
      "Previous applications df shape: (338857, 240)\n",
      "Pos-cash balance df shape: (337252, 18)\n",
      "Installments payments df shape: (339587, 24)\n",
      "Credit card balance df shape: (103558, 141)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "bureau = bureau_and_balance(num_rows)\n",
    "print(\"Bureau df shape:\", bureau.shape)\n",
    "df = df.join(bureau, how='left', on='SK_ID_CURR')\n",
    "del bureau\n",
    "gc.collect()\n",
    "\n",
    "prev = previous_applications(num_rows)\n",
    "print(\"Previous applications df shape:\", prev.shape)\n",
    "df = df.join(prev, how='left', on='SK_ID_CURR')\n",
    "del prev\n",
    "gc.collect()\n",
    "\n",
    "pos = pos_cash(num_rows)\n",
    "print(\"Pos-cash balance df shape:\", pos.shape)\n",
    "df = df.join(pos, how='left', on='SK_ID_CURR')\n",
    "del pos\n",
    "gc.collect()\n",
    "\n",
    "ins = installments_payments(num_rows)\n",
    "print(\"Installments payments df shape:\", ins.shape)\n",
    "df = df.join(ins, how='left', on='SK_ID_CURR')\n",
    "del ins\n",
    "gc.collect()\n",
    "\n",
    "cc = credit_card_balance(num_rows)\n",
    "print(\"Credit card balance df shape:\", cc.shape)\n",
    "df = df.join(cc, how='left', on='SK_ID_CURR')\n",
    "del cc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "descretize = lambda x, n: list(map(str, list(pd.qcut(x, n, duplicates='drop'))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "161"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"Level_1_stack/test_xgb-bojan-0.csv\"\n",
    "validation_file_path = 'Level_1_stack/validation_xgb-bojan-0.csv'\n",
    "num_folds = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('/media/limbo/Home-Credit/data/application_train.csv.zip', nrows= num_rows)\n",
    "n_train = train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [f for f in df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AMT_ANNUITY\n",
      "AMT_CREDIT\n",
      "AMT_GOODS_PRICE\n",
      "AMT_INCOME_TOTAL\n",
      "AMT_REQ_CREDIT_BUREAU_DAY\n",
      "AMT_REQ_CREDIT_BUREAU_HOUR\n",
      "AMT_REQ_CREDIT_BUREAU_MON\n",
      "AMT_REQ_CREDIT_BUREAU_QRT\n",
      "AMT_REQ_CREDIT_BUREAU_WEEK\n",
      "AMT_REQ_CREDIT_BUREAU_YEAR\n",
      "APARTMENTS_AVG\n",
      "APARTMENTS_MEDI\n",
      "APARTMENTS_MODE\n",
      "BASEMENTAREA_AVG\n",
      "BASEMENTAREA_MEDI\n",
      "BASEMENTAREA_MODE\n",
      "CNT_FAM_MEMBERS\n",
      "COMMONAREA_AVG\n",
      "COMMONAREA_MEDI\n",
      "COMMONAREA_MODE\n",
      "DAYS_EMPLOYED\n",
      "DAYS_LAST_PHONE_CHANGE\n",
      "DAYS_REGISTRATION\n",
      "DEF_30_CNT_SOCIAL_CIRCLE\n",
      "DEF_60_CNT_SOCIAL_CIRCLE\n",
      "ELEVATORS_AVG\n",
      "ELEVATORS_MEDI\n",
      "ELEVATORS_MODE\n",
      "ENTRANCES_AVG\n",
      "ENTRANCES_MEDI\n",
      "ENTRANCES_MODE\n",
      "EXT_SOURCE_1\n",
      "EXT_SOURCE_2\n",
      "EXT_SOURCE_3\n",
      "FLOORSMAX_AVG\n",
      "FLOORSMAX_MEDI\n",
      "FLOORSMAX_MODE\n",
      "FLOORSMIN_AVG\n",
      "FLOORSMIN_MEDI\n",
      "FLOORSMIN_MODE\n",
      "LANDAREA_AVG\n",
      "LANDAREA_MEDI\n",
      "LANDAREA_MODE\n",
      "LIVINGAPARTMENTS_AVG\n",
      "LIVINGAPARTMENTS_MEDI\n",
      "LIVINGAPARTMENTS_MODE\n",
      "LIVINGAREA_AVG\n",
      "LIVINGAREA_MEDI\n",
      "LIVINGAREA_MODE\n",
      "NONLIVINGAPARTMENTS_AVG\n",
      "NONLIVINGAPARTMENTS_MEDI\n",
      "NONLIVINGAPARTMENTS_MODE\n",
      "NONLIVINGAREA_AVG\n",
      "NONLIVINGAREA_MEDI\n",
      "NONLIVINGAREA_MODE\n",
      "OBS_30_CNT_SOCIAL_CIRCLE\n",
      "OBS_60_CNT_SOCIAL_CIRCLE\n",
      "OWN_CAR_AGE\n",
      "REGION_POPULATION_RELATIVE\n",
      "TOTALAREA_MODE\n",
      "YEARS_BEGINEXPLUATATION_AVG\n",
      "YEARS_BEGINEXPLUATATION_MEDI\n",
      "YEARS_BEGINEXPLUATATION_MODE\n",
      "YEARS_BUILD_AVG\n",
      "YEARS_BUILD_MEDI\n",
      "YEARS_BUILD_MODE\n",
      "NEW_CREDIT_TO_ANNUITY_RATIO\n",
      "NEW_CREDIT_TO_GOODS_RATIO\n",
      "NEW_DOC_IND_KURT\n",
      "NEW_INC_PER_CHLD\n",
      "NEW_INC_BY_ORG\n",
      "NEW_EMPLOY_TO_BIRTH_RATIO\n",
      "NEW_ANNUITY_TO_INCOME_RATIO\n",
      "NEW_SOURCES_PROD\n",
      "NEW_EXT_SOURCES_MEAN\n",
      "NEW_SCORES_STD\n",
      "NEW_CAR_TO_BIRTH_RATIO\n",
      "NEW_CAR_TO_EMPLOY_RATIO\n",
      "NEW_PHONE_TO_BIRTH_RATIO\n",
      "NEW_CREDIT_TO_INCOME_RATIO\n",
      "BURO_CREDIT_TYPE_Loan for the purchase of equipment_MEAN\n",
      "BURO_AMT_CREDIT_SUM_DEBT_MEAN\n",
      "BURO_AMT_CREDIT_SUM_DEBT_SUM\n",
      "BURO_CREDIT_ACTIVE_Sold_MEAN\n",
      "BURO_AMT_ANNUITY_MAX\n",
      "BURO_AMT_ANNUITY_MEAN\n",
      "BURO_CREDIT_CURRENCY_currency 4_MEAN\n",
      "BURO_CREDIT_ACTIVE_Closed_MEAN\n",
      "BURO_AMT_CREDIT_SUM_MEAN\n",
      "BURO_AMT_CREDIT_SUM_SUM\n",
      "BURO_CREDIT_TYPE_Cash loan (non-earmarked)_MEAN\n",
      "BURO_CREDIT_CURRENCY_nan_MEAN\n",
      "BURO_CREDIT_TYPE_Mortgage_MEAN\n",
      "BURO_STATUS_X_MEAN_MEAN\n",
      "BURO_STATUS_C_MEAN_MEAN\n",
      "BURO_STATUS_2_MEAN_MEAN\n",
      "BURO_CREDIT_TYPE_Real estate loan_MEAN\n",
      "BURO_CREDIT_ACTIVE_Active_MEAN\n",
      "BURO_CREDIT_CURRENCY_currency 3_MEAN\n",
      "BURO_CREDIT_TYPE_Car loan_MEAN\n",
      "BURO_DAYS_CREDIT_ENDDATE_MEAN\n",
      "BURO_CREDIT_TYPE_Loan for purchase of shares (margin lending)_MEAN\n",
      "BURO_CREDIT_TYPE_nan_MEAN\n",
      "BURO_AMT_CREDIT_SUM_OVERDUE_MEAN\n",
      "BURO_AMT_CREDIT_SUM_LIMIT_MEAN\n",
      "BURO_AMT_CREDIT_SUM_LIMIT_SUM\n",
      "BURO_MONTHS_BALANCE_MIN_MIN\n",
      "BURO_STATUS_nan_MEAN_MEAN\n",
      "BURO_DAYS_CREDIT_MEAN\n",
      "BURO_DAYS_CREDIT_VAR\n",
      "BURO_MONTHS_BALANCE_MAX_MAX\n",
      "BURO_CREDIT_DAY_OVERDUE_MEAN\n",
      "BURO_CREDIT_ACTIVE_nan_MEAN\n",
      "BURO_STATUS_0_MEAN_MEAN\n",
      "BURO_STATUS_5_MEAN_MEAN\n",
      "BURO_CREDIT_TYPE_Another type of loan_MEAN\n",
      "BURO_STATUS_1_MEAN_MEAN\n",
      "BURO_AMT_CREDIT_MAX_OVERDUE_MEAN\n",
      "BURO_MONTHS_BALANCE_SIZE_MEAN\n",
      "BURO_MONTHS_BALANCE_SIZE_SUM\n",
      "BURO_DAYS_CREDIT_UPDATE_MEAN\n",
      "BURO_STATUS_3_MEAN_MEAN\n",
      "BURO_CREDIT_TYPE_Loan for business development_MEAN\n",
      "BURO_CREDIT_TYPE_Loan for working capital replenishment_MEAN\n",
      "BURO_CREDIT_TYPE_Unknown type of loan_MEAN\n",
      "BURO_CREDIT_ACTIVE_Bad debt_MEAN\n",
      "BURO_CREDIT_TYPE_Microloan_MEAN\n",
      "BURO_CREDIT_TYPE_Credit card_MEAN\n",
      "BURO_CREDIT_CURRENCY_currency 1_MEAN\n",
      "BURO_STATUS_4_MEAN_MEAN\n",
      "BURO_CREDIT_TYPE_Interbank credit_MEAN\n",
      "BURO_CREDIT_TYPE_Consumer credit_MEAN\n",
      "BURO_CREDIT_TYPE_Mobile operator loan_MEAN\n",
      "BURO_CREDIT_CURRENCY_currency 2_MEAN\n",
      "BURO_CNT_CREDIT_PROLONG_SUM\n",
      "ACTIVE_AMT_CREDIT_SUM_DEBT_MEAN\n",
      "ACTIVE_AMT_CREDIT_SUM_DEBT_SUM\n",
      "ACTIVE_DAYS_CREDIT_ENDDATE_MEAN\n",
      "ACTIVE_AMT_ANNUITY_MAX\n",
      "ACTIVE_AMT_ANNUITY_MEAN\n",
      "ACTIVE_AMT_CREDIT_SUM_MEAN\n",
      "ACTIVE_AMT_CREDIT_SUM_SUM\n",
      "ACTIVE_AMT_CREDIT_SUM_LIMIT_MEAN\n",
      "ACTIVE_AMT_CREDIT_SUM_LIMIT_SUM\n",
      "ACTIVE_MONTHS_BALANCE_MIN_MIN\n",
      "ACTIVE_AMT_CREDIT_MAX_OVERDUE_MEAN\n",
      "ACTIVE_AMT_CREDIT_SUM_OVERDUE_MEAN\n",
      "ACTIVE_DAYS_CREDIT_MEAN\n",
      "ACTIVE_DAYS_CREDIT_VAR\n",
      "ACTIVE_MONTHS_BALANCE_SIZE_MEAN\n",
      "ACTIVE_MONTHS_BALANCE_SIZE_SUM\n",
      "ACTIVE_MONTHS_BALANCE_MAX_MAX\n",
      "ACTIVE_DAYS_CREDIT_UPDATE_MEAN\n",
      "ACTIVE_CREDIT_DAY_OVERDUE_MEAN\n",
      "ACTIVE_CNT_CREDIT_PROLONG_SUM\n",
      "CLOSED_AMT_CREDIT_SUM_DEBT_MEAN\n",
      "CLOSED_AMT_CREDIT_SUM_DEBT_SUM\n",
      "CLOSED_DAYS_CREDIT_ENDDATE_MEAN\n",
      "CLOSED_AMT_ANNUITY_MAX\n",
      "CLOSED_AMT_ANNUITY_MEAN\n",
      "CLOSED_AMT_CREDIT_SUM_MEAN\n",
      "CLOSED_AMT_CREDIT_SUM_SUM\n",
      "CLOSED_AMT_CREDIT_SUM_LIMIT_MEAN\n",
      "CLOSED_AMT_CREDIT_SUM_LIMIT_SUM\n",
      "CLOSED_MONTHS_BALANCE_MIN_MIN\n",
      "CLOSED_AMT_CREDIT_MAX_OVERDUE_MEAN\n",
      "CLOSED_AMT_CREDIT_SUM_OVERDUE_MEAN\n",
      "CLOSED_DAYS_CREDIT_MEAN\n",
      "CLOSED_DAYS_CREDIT_VAR\n",
      "CLOSED_MONTHS_BALANCE_SIZE_MEAN\n",
      "CLOSED_MONTHS_BALANCE_SIZE_SUM\n",
      "CLOSED_MONTHS_BALANCE_MAX_MAX\n",
      "CLOSED_DAYS_CREDIT_UPDATE_MEAN\n",
      "CLOSED_CREDIT_DAY_OVERDUE_MEAN\n",
      "CLOSED_CNT_CREDIT_PROLONG_SUM\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Education_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash Street: low_MEAN\n",
      "PREV_CHANNEL_TYPE_nan_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Everyday expenses_MEAN\n",
      "PREV_NAME_PAYMENT_TYPE_Non-cash from your account_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Car repairs_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Wedding / gift / holiday_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Furniture_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Jewelry_MEAN\n",
      "PREV_NAME_YIELD_GROUP_high_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_nan_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Furniture_MEAN\n",
      "PREV_CHANNEL_TYPE_Channel of corporate sales_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Card Street_MEAN\n",
      "PREV_FLAG_LAST_APPL_PER_CONTRACT_Y_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Weapon_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_nan_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Clothing and Accessories_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Furniture_MEAN\n",
      "PREV_NAME_PRODUCT_TYPE_walk-in_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Refusal to name the goal_MEAN\n",
      "PREV_NAME_CONTRACT_STATUS_Canceled_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Family_MEAN\n",
      "PREV_AMT_GOODS_PRICE_MIN\n",
      "PREV_AMT_GOODS_PRICE_MAX\n",
      "PREV_AMT_GOODS_PRICE_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Education_MEAN\n",
      "PREV_PRODUCT_COMBINATION_nan_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Business development_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Construction_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Insurance_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Mobile_MEAN\n",
      "PREV_NAME_PRODUCT_TYPE_XNA_MEAN\n",
      "PREV_CHANNEL_TYPE_Car dealer_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Buying a garage_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_WEDNESDAY_MEAN\n",
      "PREV_NAME_PORTFOLIO_POS_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS others without interest_MEAN\n",
      "PREV_DAYS_DECISION_MIN\n",
      "PREV_DAYS_DECISION_MAX\n",
      "PREV_DAYS_DECISION_MEAN\n",
      "PREV_NAME_PORTFOLIO_XNA_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_MLM partners_MEAN\n",
      "PREV_CODE_REJECT_REASON_LIMIT_MEAN\n",
      "PREV_NAME_CLIENT_TYPE_nan_MEAN\n",
      "PREV_CHANNEL_TYPE_Country-wide_MEAN\n",
      "PREV_CODE_REJECT_REASON_XNA_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Purchase of electronic equipment_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_nan_MEAN\n",
      "PREV_CNT_PAYMENT_MEAN\n",
      "PREV_CNT_PAYMENT_SUM\n",
      "PREV_NAME_GOODS_CATEGORY_Construction Materials_MEAN\n",
      "PREV_AMT_APPLICATION_MIN\n",
      "PREV_AMT_APPLICATION_MEAN\n",
      "PREV_CHANNEL_TYPE_Stone_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Computers_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_XNA_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Urgent needs_MEAN\n",
      "PREV_NAME_PORTFOLIO_Cars_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_XNA_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Buying a holiday home / land_MEAN\n",
      "PREV_NAME_PRODUCT_TYPE_x-sell_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash Street: middle_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_MONDAY_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Unaccompanied_MEAN\n",
      "PREV_NAME_CONTRACT_TYPE_Cash loans_MEAN\n",
      "PREV_CHANNEL_TYPE_Regional / Local_MEAN\n",
      "PREV_NAME_PORTFOLIO_nan_MEAN\n",
      "PREV_NAME_CONTRACT_STATUS_Refused_MEAN\n",
      "PREV_NAME_PORTFOLIO_Cash_MEAN\n",
      "PREV_NAME_YIELD_GROUP_low_normal_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Building a house or an annex_MEAN\n",
      "PREV_AMT_DOWN_PAYMENT_MIN\n",
      "PREV_AMT_DOWN_PAYMENT_MAX\n",
      "PREV_AMT_DOWN_PAYMENT_MEAN\n",
      "PREV_CODE_REJECT_REASON_SCOFR_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Vehicles_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Additional Service_MEAN\n",
      "PREV_CHANNEL_TYPE_Contact center_MEAN\n",
      "PREV_NAME_YIELD_GROUP_nan_MEAN\n",
      "PREV_NAME_CONTRACT_STATUS_Approved_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_XAP_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash X-Sell: middle_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_nan_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Gardening_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS industry with interest_MEAN\n",
      "PREV_NAME_PORTFOLIO_Cards_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Gasification / water supply_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS household with interest_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Medicine_MEAN\n",
      "PREV_NAME_YIELD_GROUP_low_action_MEAN\n",
      "PREV_NAME_PAYMENT_TYPE_XNA_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Journey_MEAN\n",
      "PREV_NAME_CONTRACT_TYPE_nan_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Consumer Electronics_MEAN\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PREV_NAME_PAYMENT_TYPE_nan_MEAN\n",
      "PREV_NAME_CLIENT_TYPE_Refreshed_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Office Appliances_MEAN\n",
      "PREV_CODE_REJECT_REASON_VERIF_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_SATURDAY_MEAN\n",
      "PREV_CODE_REJECT_REASON_CLIENT_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Direct Sales_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Buying a new car_MEAN\n",
      "PREV_CODE_REJECT_REASON_HC_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Animals_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Auto technology_MEAN\n",
      "PREV_NAME_CLIENT_TYPE_Repeater_MEAN\n",
      "PREV_CHANNEL_TYPE_AP+ (Cash loan)_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Other_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Buying a used car_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Children_MEAN\n",
      "PREV_NAME_CONTRACT_TYPE_XNA_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_FRIDAY_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Auto Accessories_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Consumer electronics_MEAN\n",
      "PREV_CODE_REJECT_REASON_nan_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Clothing_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Tourism_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Medical Supplies_MEAN\n",
      "PREV_APP_CREDIT_PERC_MIN\n",
      "PREV_APP_CREDIT_PERC_MAX\n",
      "PREV_APP_CREDIT_PERC_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Medicine_MEAN\n",
      "PREV_NAME_PAYMENT_TYPE_Cash through the bank_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS mobile without interest_MEAN\n",
      "PREV_NAME_TYPE_SUITE_nan_MEAN\n",
      "PREV_NAME_YIELD_GROUP_middle_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Tourism_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS other with interest_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Other_A_MEAN\n",
      "PREV_NAME_PAYMENT_TYPE_Cashless from the account of the employer_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_SUNDAY_MEAN\n",
      "PREV_CODE_REJECT_REASON_SYSTEM_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Money for a third person_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Homewares_MEAN\n",
      "PREV_FLAG_LAST_APPL_PER_CONTRACT_nan_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS mobile with interest_MEAN\n",
      "PREV_AMT_ANNUITY_MAX\n",
      "PREV_AMT_ANNUITY_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Spouse, partner_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_THURSDAY_MEAN\n",
      "PREV_NAME_CLIENT_TYPE_New_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Repairs_MEAN\n",
      "PREV_FLAG_LAST_APPL_PER_CONTRACT_N_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash X-Sell: high_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS industry without interest_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_XNA_MEAN\n",
      "PREV_PRODUCT_COMBINATION_POS household without interest_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Audio/Video_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Hobby_MEAN\n",
      "PREV_NAME_CONTRACT_STATUS_Unused offer_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Jewelry_MEAN\n",
      "PREV_CODE_REJECT_REASON_XAP_MEAN\n",
      "PREV_NAME_CONTRACT_TYPE_Consumer loans_MEAN\n",
      "PREV_CODE_REJECT_REASON_SCO_MEAN\n",
      "PREV_WEEKDAY_APPR_PROCESS_START_TUESDAY_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash X-Sell: low_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Payments on other loans_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Card X-Sell_MEAN\n",
      "PREV_HOUR_APPR_PROCESS_START_MIN\n",
      "PREV_HOUR_APPR_PROCESS_START_MAX\n",
      "PREV_HOUR_APPR_PROCESS_START_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Other_B_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Other_MEAN\n",
      "PREV_CHANNEL_TYPE_Credit and cash offices_MEAN\n",
      "PREV_NAME_PRODUCT_TYPE_nan_MEAN\n",
      "PREV_NAME_CLIENT_TYPE_XNA_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Connectivity_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash Street: high_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Photo / Cinema Equipment_MEAN\n",
      "PREV_AMT_CREDIT_MIN\n",
      "PREV_AMT_CREDIT_MAX\n",
      "PREV_AMT_CREDIT_MEAN\n",
      "PREV_NAME_CASH_LOAN_PURPOSE_Buying a home_MEAN\n",
      "PREV_NAME_CONTRACT_STATUS_nan_MEAN\n",
      "PREV_NAME_TYPE_SUITE_Group of people_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Sport and Leisure_MEAN\n",
      "PREV_NAME_CONTRACT_TYPE_Revolving loans_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_House Construction_MEAN\n",
      "PREV_RATE_DOWN_PAYMENT_MIN\n",
      "PREV_RATE_DOWN_PAYMENT_MAX\n",
      "PREV_RATE_DOWN_PAYMENT_MEAN\n",
      "PREV_NAME_GOODS_CATEGORY_Fitness_MEAN\n",
      "PREV_PRODUCT_COMBINATION_Cash_MEAN\n",
      "PREV_NAME_SELLER_INDUSTRY_Industry_MEAN\n",
      "PREV_NAME_YIELD_GROUP_XNA_MEAN\n",
      "APPROVED_RATE_DOWN_PAYMENT_MIN\n",
      "APPROVED_RATE_DOWN_PAYMENT_MAX\n",
      "APPROVED_RATE_DOWN_PAYMENT_MEAN\n",
      "APPROVED_HOUR_APPR_PROCESS_START_MIN\n",
      "APPROVED_HOUR_APPR_PROCESS_START_MAX\n",
      "APPROVED_HOUR_APPR_PROCESS_START_MEAN\n",
      "APPROVED_AMT_APPLICATION_MIN\n",
      "APPROVED_AMT_APPLICATION_MEAN\n",
      "APPROVED_AMT_CREDIT_MIN\n",
      "APPROVED_AMT_CREDIT_MAX\n",
      "APPROVED_AMT_CREDIT_MEAN\n",
      "APPROVED_DAYS_DECISION_MIN\n",
      "APPROVED_DAYS_DECISION_MAX\n",
      "APPROVED_DAYS_DECISION_MEAN\n",
      "APPROVED_AMT_ANNUITY_MAX\n",
      "APPROVED_AMT_ANNUITY_MEAN\n",
      "APPROVED_AMT_GOODS_PRICE_MIN\n",
      "APPROVED_AMT_GOODS_PRICE_MAX\n",
      "APPROVED_AMT_GOODS_PRICE_MEAN\n",
      "APPROVED_AMT_DOWN_PAYMENT_MIN\n",
      "APPROVED_AMT_DOWN_PAYMENT_MAX\n",
      "APPROVED_AMT_DOWN_PAYMENT_MEAN\n",
      "APPROVED_CNT_PAYMENT_MEAN\n",
      "APPROVED_CNT_PAYMENT_SUM\n",
      "APPROVED_APP_CREDIT_PERC_MIN\n",
      "APPROVED_APP_CREDIT_PERC_MAX\n",
      "APPROVED_APP_CREDIT_PERC_MEAN\n",
      "REFUSED_RATE_DOWN_PAYMENT_MIN\n",
      "REFUSED_RATE_DOWN_PAYMENT_MAX\n",
      "REFUSED_RATE_DOWN_PAYMENT_MEAN\n",
      "REFUSED_HOUR_APPR_PROCESS_START_MIN\n",
      "REFUSED_HOUR_APPR_PROCESS_START_MAX\n",
      "REFUSED_HOUR_APPR_PROCESS_START_MEAN\n",
      "REFUSED_AMT_APPLICATION_MIN\n",
      "REFUSED_AMT_APPLICATION_MEAN\n",
      "REFUSED_AMT_CREDIT_MIN\n",
      "REFUSED_AMT_CREDIT_MAX\n",
      "REFUSED_AMT_CREDIT_MEAN\n",
      "REFUSED_DAYS_DECISION_MIN\n",
      "REFUSED_DAYS_DECISION_MAX\n",
      "REFUSED_DAYS_DECISION_MEAN\n",
      "REFUSED_AMT_ANNUITY_MAX\n",
      "REFUSED_AMT_ANNUITY_MEAN\n",
      "REFUSED_AMT_GOODS_PRICE_MIN\n",
      "REFUSED_AMT_GOODS_PRICE_MAX\n",
      "REFUSED_AMT_GOODS_PRICE_MEAN\n",
      "REFUSED_AMT_DOWN_PAYMENT_MIN\n",
      "REFUSED_AMT_DOWN_PAYMENT_MAX\n",
      "REFUSED_AMT_DOWN_PAYMENT_MEAN\n",
      "REFUSED_CNT_PAYMENT_MEAN\n",
      "REFUSED_CNT_PAYMENT_SUM\n",
      "REFUSED_APP_CREDIT_PERC_MIN\n",
      "REFUSED_APP_CREDIT_PERC_MAX\n",
      "REFUSED_APP_CREDIT_PERC_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_nan_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Completed_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Canceled_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Amortized debt_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Active_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_XNA_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Returned to the store_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Signed_MEAN\n",
      "POS_SK_DPD_DEF_MAX\n",
      "POS_SK_DPD_DEF_MEAN\n",
      "POS_NAME_CONTRACT_STATUS_Demand_MEAN\n",
      "POS_MONTHS_BALANCE_MAX\n",
      "POS_MONTHS_BALANCE_MEAN\n",
      "POS_MONTHS_BALANCE_SIZE\n",
      "POS_NAME_CONTRACT_STATUS_Approved_MEAN\n",
      "POS_SK_DPD_MAX\n",
      "POS_SK_DPD_MEAN\n",
      "POS_COUNT\n",
      "INSTAL_AMT_INSTALMENT_MAX\n",
      "INSTAL_AMT_INSTALMENT_MEAN\n",
      "INSTAL_AMT_INSTALMENT_SUM\n",
      "INSTAL_PAYMENT_PERC_MEAN\n",
      "INSTAL_PAYMENT_PERC_SUM\n",
      "INSTAL_PAYMENT_PERC_VAR\n",
      "INSTAL_DBD_MAX\n",
      "INSTAL_DBD_MEAN\n",
      "INSTAL_DBD_SUM\n",
      "INSTAL_AMT_PAYMENT_MIN\n",
      "INSTAL_AMT_PAYMENT_MAX\n",
      "INSTAL_AMT_PAYMENT_MEAN\n",
      "INSTAL_AMT_PAYMENT_SUM\n",
      "INSTAL_DPD_MAX\n",
      "INSTAL_DPD_MEAN\n",
      "INSTAL_DPD_SUM\n",
      "INSTAL_PAYMENT_DIFF_MEAN\n",
      "INSTAL_PAYMENT_DIFF_SUM\n",
      "INSTAL_PAYMENT_DIFF_VAR\n",
      "INSTAL_NUM_INSTALMENT_VERSION_NUNIQUE\n",
      "INSTAL_DAYS_ENTRY_PAYMENT_MAX\n",
      "INSTAL_DAYS_ENTRY_PAYMENT_MEAN\n",
      "INSTAL_DAYS_ENTRY_PAYMENT_SUM\n",
      "INSTAL_COUNT\n",
      "CC_MONTHS_BALANCE_MIN\n",
      "CC_MONTHS_BALANCE_MAX\n",
      "CC_MONTHS_BALANCE_MEAN\n",
      "CC_MONTHS_BALANCE_SUM\n",
      "CC_MONTHS_BALANCE_VAR\n",
      "CC_AMT_BALANCE_MIN\n",
      "CC_AMT_BALANCE_MAX\n",
      "CC_AMT_BALANCE_MEAN\n",
      "CC_AMT_BALANCE_SUM\n",
      "CC_AMT_BALANCE_VAR\n",
      "CC_AMT_CREDIT_LIMIT_ACTUAL_MIN\n",
      "CC_AMT_CREDIT_LIMIT_ACTUAL_MAX\n",
      "CC_AMT_CREDIT_LIMIT_ACTUAL_MEAN\n",
      "CC_AMT_CREDIT_LIMIT_ACTUAL_SUM\n",
      "CC_AMT_CREDIT_LIMIT_ACTUAL_VAR\n",
      "CC_AMT_DRAWINGS_ATM_CURRENT_MIN\n",
      "CC_AMT_DRAWINGS_ATM_CURRENT_MAX\n",
      "CC_AMT_DRAWINGS_ATM_CURRENT_MEAN\n",
      "CC_AMT_DRAWINGS_ATM_CURRENT_SUM\n",
      "CC_AMT_DRAWINGS_ATM_CURRENT_VAR\n",
      "CC_AMT_DRAWINGS_CURRENT_MIN\n",
      "CC_AMT_DRAWINGS_CURRENT_MAX\n",
      "CC_AMT_DRAWINGS_CURRENT_MEAN\n",
      "CC_AMT_DRAWINGS_CURRENT_SUM\n",
      "CC_AMT_DRAWINGS_CURRENT_VAR\n",
      "CC_AMT_DRAWINGS_OTHER_CURRENT_MIN\n",
      "CC_AMT_DRAWINGS_OTHER_CURRENT_MAX\n",
      "CC_AMT_DRAWINGS_OTHER_CURRENT_MEAN\n",
      "CC_AMT_DRAWINGS_OTHER_CURRENT_SUM\n",
      "CC_AMT_DRAWINGS_OTHER_CURRENT_VAR\n",
      "CC_AMT_DRAWINGS_POS_CURRENT_MIN\n",
      "CC_AMT_DRAWINGS_POS_CURRENT_MAX\n",
      "CC_AMT_DRAWINGS_POS_CURRENT_MEAN\n",
      "CC_AMT_DRAWINGS_POS_CURRENT_SUM\n",
      "CC_AMT_DRAWINGS_POS_CURRENT_VAR\n",
      "CC_AMT_INST_MIN_REGULARITY_MIN\n",
      "CC_AMT_INST_MIN_REGULARITY_MAX\n",
      "CC_AMT_INST_MIN_REGULARITY_MEAN\n",
      "CC_AMT_INST_MIN_REGULARITY_SUM\n",
      "CC_AMT_INST_MIN_REGULARITY_VAR\n",
      "CC_AMT_PAYMENT_CURRENT_MIN\n",
      "CC_AMT_PAYMENT_CURRENT_MAX\n",
      "CC_AMT_PAYMENT_CURRENT_MEAN\n",
      "CC_AMT_PAYMENT_CURRENT_SUM\n",
      "CC_AMT_PAYMENT_CURRENT_VAR\n",
      "CC_AMT_PAYMENT_TOTAL_CURRENT_MIN\n",
      "CC_AMT_PAYMENT_TOTAL_CURRENT_MAX\n",
      "CC_AMT_PAYMENT_TOTAL_CURRENT_MEAN\n",
      "CC_AMT_PAYMENT_TOTAL_CURRENT_SUM\n",
      "CC_AMT_PAYMENT_TOTAL_CURRENT_VAR\n",
      "CC_AMT_RECEIVABLE_PRINCIPAL_MIN\n",
      "CC_AMT_RECEIVABLE_PRINCIPAL_MAX\n",
      "CC_AMT_RECEIVABLE_PRINCIPAL_MEAN\n",
      "CC_AMT_RECEIVABLE_PRINCIPAL_SUM\n",
      "CC_AMT_RECEIVABLE_PRINCIPAL_VAR\n",
      "CC_AMT_RECIVABLE_MIN\n",
      "CC_AMT_RECIVABLE_MAX\n",
      "CC_AMT_RECIVABLE_MEAN\n",
      "CC_AMT_RECIVABLE_SUM\n",
      "CC_AMT_RECIVABLE_VAR\n",
      "CC_AMT_TOTAL_RECEIVABLE_MIN\n",
      "CC_AMT_TOTAL_RECEIVABLE_MAX\n",
      "CC_AMT_TOTAL_RECEIVABLE_MEAN\n",
      "CC_AMT_TOTAL_RECEIVABLE_SUM\n",
      "CC_AMT_TOTAL_RECEIVABLE_VAR\n",
      "CC_CNT_DRAWINGS_ATM_CURRENT_MIN\n",
      "CC_CNT_DRAWINGS_ATM_CURRENT_MAX\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CC_CNT_DRAWINGS_ATM_CURRENT_MEAN\n",
      "CC_CNT_DRAWINGS_ATM_CURRENT_SUM\n",
      "CC_CNT_DRAWINGS_ATM_CURRENT_VAR\n",
      "CC_CNT_DRAWINGS_CURRENT_MIN\n",
      "CC_CNT_DRAWINGS_CURRENT_MAX\n",
      "CC_CNT_DRAWINGS_CURRENT_MEAN\n",
      "CC_CNT_DRAWINGS_CURRENT_SUM\n",
      "CC_CNT_DRAWINGS_CURRENT_VAR\n",
      "CC_CNT_DRAWINGS_OTHER_CURRENT_MIN\n",
      "CC_CNT_DRAWINGS_OTHER_CURRENT_MAX\n",
      "CC_CNT_DRAWINGS_OTHER_CURRENT_MEAN\n",
      "CC_CNT_DRAWINGS_OTHER_CURRENT_SUM\n",
      "CC_CNT_DRAWINGS_OTHER_CURRENT_VAR\n",
      "CC_CNT_DRAWINGS_POS_CURRENT_MIN\n",
      "CC_CNT_DRAWINGS_POS_CURRENT_MAX\n",
      "CC_CNT_DRAWINGS_POS_CURRENT_MEAN\n",
      "CC_CNT_DRAWINGS_POS_CURRENT_SUM\n",
      "CC_CNT_DRAWINGS_POS_CURRENT_VAR\n",
      "CC_CNT_INSTALMENT_MATURE_CUM_MIN\n",
      "CC_CNT_INSTALMENT_MATURE_CUM_MAX\n",
      "CC_CNT_INSTALMENT_MATURE_CUM_MEAN\n",
      "CC_CNT_INSTALMENT_MATURE_CUM_SUM\n",
      "CC_CNT_INSTALMENT_MATURE_CUM_VAR\n",
      "CC_SK_DPD_MIN\n",
      "CC_SK_DPD_MAX\n",
      "CC_SK_DPD_MEAN\n",
      "CC_SK_DPD_SUM\n",
      "CC_SK_DPD_VAR\n",
      "CC_SK_DPD_DEF_MIN\n",
      "CC_SK_DPD_DEF_MAX\n",
      "CC_SK_DPD_DEF_MEAN\n",
      "CC_SK_DPD_DEF_SUM\n",
      "CC_SK_DPD_DEF_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Active_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Active_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Active_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Active_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Active_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Approved_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Approved_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Approved_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Approved_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Approved_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Completed_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Completed_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Completed_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Completed_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Completed_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Demand_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Demand_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Demand_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Demand_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Demand_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Refused_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Refused_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Refused_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Refused_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Refused_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Sent proposal_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Sent proposal_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Sent proposal_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Sent proposal_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Sent proposal_VAR\n",
      "CC_NAME_CONTRACT_STATUS_Signed_MIN\n",
      "CC_NAME_CONTRACT_STATUS_Signed_MAX\n",
      "CC_NAME_CONTRACT_STATUS_Signed_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_Signed_SUM\n",
      "CC_NAME_CONTRACT_STATUS_Signed_VAR\n",
      "CC_NAME_CONTRACT_STATUS_nan_MIN\n",
      "CC_NAME_CONTRACT_STATUS_nan_MAX\n",
      "CC_NAME_CONTRACT_STATUS_nan_MEAN\n",
      "CC_NAME_CONTRACT_STATUS_nan_SUM\n",
      "CC_NAME_CONTRACT_STATUS_nan_VAR\n",
      "CC_COUNT\n"
     ]
    }
   ],
   "source": [
    "lencoder = LabelEncoder()\n",
    "for feature in feats:\n",
    "    if df[feature].dtype == 'float64':\n",
    "        print(feature)\n",
    "        try:\n",
    "            df[feature] = lencoder.fit_transform(descretize(df[feature].values, 100))\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = df[0:n_train]\n",
    "test_df = df[n_train:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting LightGBM. Train shape: (307511, 776), test shape: (48744, 776)\n",
      "(307511, 773)\n",
      "(246008, 773) (61503, 773) (48744, 773)\n",
      "Fold  1 AUC : 0.778573\n",
      "(246009, 773) (61502, 773) (48744, 773)\n",
      "Fold  2 AUC : 0.778729\n",
      "(246009, 773) (61502, 773) (48744, 773)\n",
      "Fold  3 AUC : 0.772564\n",
      "(246009, 773) (61502, 773) (48744, 773)\n",
      "Fold  4 AUC : 0.775938\n",
      "(246009, 773) (61502, 773) (48744, 773)\n",
      "Fold  5 AUC : 0.779885\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoding = 'ohe'\n",
    "\n",
    "\n",
    "print(\"Starting LightGBM. Train shape: {}, test shape: {}\".format(train_df.shape, test_df.shape))\n",
    "gc.collect()\n",
    "# Cross validation model\n",
    "folds = KFold(n_splits=num_folds, shuffle=True, random_state=1001)\n",
    "# Create arrays and dataframes to store results\n",
    "oof_preds = np.zeros(train_df.shape[0])\n",
    "sub_preds = np.zeros(test_df.shape[0])\n",
    "\n",
    "feats = [f for f in train_df.columns if f not in ['TARGET','SK_ID_CURR','SK_ID_BUREAU','SK_ID_PREV','index']]\n",
    "\n",
    "#feats = [col for col in feats_0 if df[col].dtype == 'object']\n",
    "\n",
    "\n",
    "print(train_df[feats].shape)\n",
    "for n_fold, (train_idx, valid_idx) in enumerate(folds.split(train_df[feats], train_df['TARGET'])):\n",
    "        \n",
    "      \n",
    "        categorical_columns = [col for col in train_df.columns if train_df[col].dtype == 'object']\n",
    "        \n",
    "        if encoding == 'ohe':\n",
    "            \n",
    "            enc = ce.TargetEncoder(impute_missing=True, cols=categorical_columns).fit(train_df[feats].iloc[train_idx],\n",
    "                                                                                       train_df['TARGET'].iloc[train_idx])\n",
    "            x_train = enc.transform(train_df[feats].iloc[train_idx]).fillna(-999)\n",
    "            x_valid = enc.transform(train_df[feats].iloc[valid_idx]).fillna(-999)\n",
    "            x_test = enc.transform(test_df[feats]).fillna(-999)\n",
    "            print(x_train.shape, x_valid.shape, x_test.shape)\n",
    "\n",
    "            dtest = xgb.DMatrix(x_test.values)\n",
    "            dtrain = xgb.DMatrix(x_train.values, \n",
    "                         label=train_df['TARGET'].iloc[train_idx].values)\n",
    "            dvalid = xgb.DMatrix(x_valid.values, \n",
    "                         train_df['TARGET'].iloc[valid_idx].values)\n",
    "\n",
    "        params = dict(\n",
    "          booster=\"gbtree\",\n",
    "          eval_metric = \"auc\",\n",
    "          nthread=4,\n",
    "          eta=0.05,\n",
    "          max_depth=6,\n",
    "          min_child_weight = 30,\n",
    "          gamma=0,\n",
    "          subsample = 0.85,\n",
    "          colsample_bytree = 0.7,\n",
    "          colsample_bylevel = 0.632,\n",
    "          alpha=0)\n",
    "\n",
    "        clf = xgb.train(\n",
    "            params,\n",
    "            dtrain,\n",
    "            num_boost_round=30000,\n",
    "            evals=[(dtrain, 'train'), (dvalid, 'valid')],\n",
    "            early_stopping_rounds=100,\n",
    "            verbose_eval=False\n",
    "        )\n",
    "        \n",
    "        oof_preds[valid_idx] = clf.predict(dvalid)\n",
    "        sub_preds += clf.predict(dtest) / folds.n_splits\n",
    "\n",
    "        print('Fold %2d AUC : %.6f' % (n_fold + 1, roc_auc_score(train_df['TARGET'].iloc[valid_idx].values, oof_preds[valid_idx])))\n",
    "        del clf, dtrain, dvalid\n",
    "        gc.collect()\n",
    "\n",
    "# print('Full AUC score %.6f' % roc_auc_score(train_df['TARGET'].iloc[train_idx].values, oof_preds))\n",
    "# # Write submission file and plot feature importance\n",
    "\n",
    "sub_df = test_df[['SK_ID_CURR']].copy()\n",
    "sub_df['TARGET'] = sub_preds\n",
    "sub_df[['SK_ID_CURR', 'TARGET']].to_csv(test_file_path, index= False)\n",
    "\n",
    "\n",
    "val_df = train_df[['SK_ID_CURR', 'TARGET']].copy()\n",
    "val_df['TARGET'] = oof_preds\n",
    "val_df[['SK_ID_CURR', 'TARGET']].to_csv(validation_file_path, index= False)        \n",
    "            \n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>0.672237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0.117377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0.157362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0.182700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0.329907</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR    TARGET\n",
       "0      100002  0.672237\n",
       "1      100003  0.117377\n",
       "2      100004  0.157362\n",
       "3      100006  0.182700\n",
       "4      100007  0.329907"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RangeIndex(start=0, stop=307511, step=1)"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_df.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(356255, 776)"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
